#coding:utf-8
"""
*   TG_Coder : Dzlua
*   Email    : 505544956@qq.com
*   Time     : 2017/09/08
"""

import sys
sys.path.append("..")

from database import DataBase
import codecs

import string, sys, os

from progressbar import *

'''
    1-> get path
    2-> copy to 4T
    3-> unpack
    4-> del binary
    5-> pack
'''

class ToUdisk(DataBase):
    def __init__(self, host, user, pwd, db):
        self.__tb = 'toudisk';
        DataBase.__init__(self, host, user, pwd, db)
        sql = "CREATE TABLE IF NOT EXISTS " + self.__tb + " " \
            "(id INT AUTO_INCREMENT PRIMARY KEY," \
            " name VARCHAR(64) NOT NULL," \
            " type VARCHAR(256) NOT NULL DEFAULT ''," \
            " src TEXT," \
            " dst TEXT," \
            " add_date TIMESTAMP," \
            " UNIQUE(name) )" \
            " DEFAULT CHARSET=utf8;"
        DataBase.execute(self, sql)

        self.__src1 = '/home/zhouhan/dzlua/tg-data'
        self.__src2 = '/home/zhouhan/dzlua/tg-data2'
        self._src_root = '/home/zhouhan/dzlua/tg-data-back/tg-data/src'
        self._dst_root = '/home/zhouhan/dzlua/tg-data-back/tg-data/tar'

    def start(self, bid, eid):
        self.read(self.__on_read, [bid, eid])
    def write(self, text):
        f = codecs.open(self.__tb + '.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def _select(self, conn, sql):
        cursor = conn.cursor()
        cursor.execute(sql)
        result = cursor.fetchall()
        cursor.close()
        return result
    def _getCount(self, conn):
        sql = 'SELECT count(*) FROM librarys;'
        result = self._select(conn, sql)

        count = 0
        for r in result:
            count = r[0]
            break
        return count
    def _getPath(self, data):
        path = data['path'].encode('utf8')
        name = data['name'].encode('utf8')

        abs =  '/' + path + '/' + name
        path1 = self.__src1 + abs
        path2 = self.__src2 + abs
        return path1, path2, name, path
    def _getPathdst(self, name, path):
        path1 = self._src_root + '/' + path + '/' + name
        path2 = self._dst_root + '/' + path + '/' + name
        return path1, path2
    def _checkfile(self, file):
        if os.path.isfile(file):
            return os.path.getsize(file)
        return False
    def _checkdir(self, dir):
        if not os.path.isdir(dir):
            return False
        if os.listdir(dir):
            return self._getdirsize(dir)
        return False
    def _getdirsize(self, dir):
        size = 0l
        for (root,dirs,files) in os.walk(dir):
            for name in files:
                try:
                    size += os.path.getsize(os.path.join(root,name))
                except:
                    continue
        return size
    def mkdirs(self, path, dir=False):
        pt = path
        try:
            if dir:
                os.makedirs(path)
                return
            pt = path[0 : path.rfind('/')]
            os.makedirs(pt)
        except:
            print('[ToUdisk] mkdirs error,(exist?): %s' % pt)
    def copy(self, src, dst):
        cmd = 'cp -rf %s %s' % (src, dst)
        print '[ToUdisk] copy : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        print '[ToUdisk] copy : done!'
    def deldirfile(self, path):
        try:
            cmd = 'rm -rf %s' % path
            os.system(cmd)
        except:
            print('[ToUdisk] deldirfile error,(?): %s' % path)
    def istextfile(self, filename, blocksize = 512):
        if os.path.isfile(filename) == False:
            return False
        return self.istext(open(filename).read(blocksize))
    def istext(self, s):
        text_characters = "".join(map(chr, range(32, 127)) + list("\n\r\t\b"))
        _null_trans = string.maketrans("", "")

        if "\0" in s:
            return False
    
        if not s:
            return False
        t = s.translate(_null_trans, text_characters)
        if float(len(t))/float(len(s)) > 0.30:
            return False
        return True
    def _checkdelfiles(self, path):
        def _delfile(file):
            bt = self.istextfile(file)
            if not bt:
                self.deldirfile(file)
        #
        for (root,dirs,files) in os.walk(path):
            for file in files:
                _delfile(root + '/' + file)
            for dir in dirs:
                self._checkdelfiles(root + dir)
    def unpack_zip(self, file, dst_dir):
        #del dir
        self.deldirfile(dst_dir)
        #create dir
        self.mkdirs(dst_dir, True)
        #unzip
        cmd = 'unzip -q %s -d %s' % (file, dst_dir)
        print '[ToUdisk] unpack zip : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        print '[ToUdisk] unpack zip : done!'
    def unpack_tar(self, file, dst_dir):
        #del dir
        self.deldirfile(dst_dir)
        #create dir
        self.mkdirs(dst_dir, True)
        #untar
        cmd = 'tar -zxf %s -C %s' % (file, dst_dir)
        print '[ToUdisk] unpack tar : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        print '[ToUdisk] unpack tar : done!'
    def pack_tar(self, src_dir, dst_dir, name):
        file = dst_dir + '.tar.gz'
        cmd = 'tar -zcf %s -C %s %s' % (file, src_dir + '/..', name)
        print '[ToUdisk] pack tar : %s' % cmd
        ret = ( os.system(cmd) == 0 )
        assert(ret)
        print '[ToUdisk] pack tar : done!'
        return name + '.tar.gz'
    def __on_read(self, conn, args):
        bid = args[0]
        eid = args[1]

        count = self._getCount(conn)
        id = bid
        while id < count and id <= eid:
            print('------------------------------')
            print('[ToUdisk] id:%s/%d[%s,%s], num:%s' % (id, count, bid, eid, id-bid+1) )
            sql = "SELECT id,name,local_path,type FROM librarys WHERE id=%s;" % id
            result = self._select(conn, sql)
            for r in result:
                info = {
                    'id': r[0],
                    'name': r[1],
                    'path': r[2],
                    'type': r[3],
                }
                try:
                    self._getPjo(info)
                except:
                    errmsg = '[ToUdisk] ERROR id:%s,name:%s' % (id, info['name'])
                    self.write(errmsg + '\n')
                    print(errmsg)
                self._getPjo(info)
                break
            id = id +1
    def __on_save(self, conn, data, args):
        try:
            cursor = conn.cursor()
            sql = "INSERT INTO " + self.__tb + " (name,type,src,dst)" \
                " VALUES(%s,%s,%s,%s);"
            param = ( data['name'], data['type'],data['src'],data['dst'] )
            cursor.execute(sql, param)
            cursor.close()
            conn.commit()
        except conn.Error, e:
            err = '[ToUdisk] Error __on_save : %d %s, %s ' % (e[0], e[1], data)
            self.write(err + '\n')
            print(err)
        finally:
            print('[ToUdisk] save id: %d, name: %s' % (data['id'], data['name']))
    def _getPjo(self, data):
        def _file(src):
            zip = src + '.zip'
            tar = src + '.tar.gz'
            git = src
            return zip, tar, git
        #
        src1,src2,name,path = self._getPath(data)
        for src in [src1, src2]:
            zip,tar,git = _file(src)
            
            #
            res = self._checkfile(zip)
            if res:
                if self._zip(res, zip, name, '.zip', path, data):
                    break
            #
            res = self._checkfile(tar)
            if res:
                if self._tar(res, tar, name, '.tar.gz', path, data):
                    break
            #
            res = self._checkdir(git)
            if res:
                if self._git(res, git, name, '', path, data):
                    break
    def _zip(self, res, src, name, suffix, path, data):
        print '[ToUdisk] zip: %s, src: %s, name: %s, suffix: %s, path:%s' % (res, src, name, suffix, path)

        dst_src,dst_tar = self._getPathdst(name, path)
        self.mkdirs(dst_src)
        self.copy(src, dst_src + suffix)
        self.unpack_zip(src, dst_tar)
        print('[ToUdisk] checking binary : ...')
        self._checkdelfiles(dst_tar)
        print('[ToUdisk] checking binary : done')
        filename = self.pack_tar(dst_tar, dst_tar, name)
        self.deldirfile(dst_tar)

        data['src'] = (name + suffix).decode('utf8')
        data['dst'] = filename.decode('utf8')

        self.save(self.__on_save, data)

        return True
    def _tar(self, res, src, name, suffix, path, data):
        print '[ToUdisk] tar: %s, src: %s, name: %s, suffix: %s, path:%s' % (res, src, name, suffix, path)

        dst_src,dst_tar = self._getPathdst(name, path)
        self.mkdirs(dst_src)
        self.copy(src, dst_src + suffix)
        self.unpack_tar(src, dst_tar)
        print('[ToUdisk] checking binary : ...')
        self._checkdelfiles(dst_tar)
        print('[ToUdisk] checking binary : done')
        filename = self.pack_tar(dst_tar, dst_tar, name)
        self.deldirfile(dst_tar)

        data['src'] = (name + suffix).decode('utf8')
        data['dst'] = filename.decode('utf8')

        self.save(self.__on_save, data)

        return True
    def _git(self, res, src, name, suffix, path, data):
        print '[ToUdisk] git: %s, src: %s, name: %s, suffix: %s, path:%s' % (res, src, name, suffix, path)

        dst_src,dst_tar = self._getPathdst(name, path)
        self.mkdirs(dst_src)
        self.mkdirs(dst_tar)
        self.copy(src, dst_src + suffix)
        print('[ToUdisk] checking binary : ...')
        self._checkdelfiles(dst_tar)
        print('[ToUdisk] checking binary : done')
        filename = self.pack_tar(dst_src, dst_tar, name)

        data['src'] = (name + suffix).decode('utf8')
        data['dst'] = filename.decode('utf8')

        self.save(self.__on_save, data)

        return True
#-------------------------------#
todisk = ToUdisk('localhost', 'dzlua', 'dzlua', 'spider')
todisk.start(0, 10310)
todisk.close()